﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web;
using TaoBao.Util;

namespace Taobao.Util
{
    public class SpiderMethod
    {
        #region 根据url获取html页面代码
        /// <summary>
        /// 根据url获取页面信息
        /// </summary>
        /// <param name="WebUrl"></param>
        /// <returns></returns>
        public static string GetWebHTML(string WebUrl)
        {
            //test
            string strUrl = WebUrl;
            Encoding encoding = System.Text.Encoding.Default;
            WebRequest request = WebRequest.Create(strUrl);
            request.Timeout = 20000;
            //request.Headers.Set("Pragma", "no-cache");
            //request.Method = "POST"; 
            request.Credentials = CredentialCache.DefaultCredentials;
            string responseFromServer = "";
            string sException = null;
            try
            {
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();

                if (response.StatusDescription.ToUpper() == "OK")
                {
                    switch (response.CharacterSet.ToLower())
                    {
                        case "gbk":
                            encoding = Encoding.GetEncoding("GBK");//貌似用GB2312就可以
                            break;
                        case "gb2312":
                            encoding = Encoding.GetEncoding("GB2312");
                            break;
                        case "utf-8":
                            encoding = Encoding.UTF8;
                            break;
                        case "big5":
                            encoding = Encoding.GetEncoding("Big5");
                            break;
                        case "iso-8859-1":
                            encoding = Encoding.UTF8;//ISO-8859-1的编码用UTF-8处理，致少优酷的是这种方法没有乱码
                            break;
                        default:
                            encoding = Encoding.UTF8;//如果分析不出来就用的UTF-8
                            break;
                    }
                    string item = "Lenght:" + response.ContentLength.ToString() + "<br>CharacterSet:" + response.CharacterSet + "<br>Headers:" + response.Headers + "<br>";
                    Stream dataStream = response.GetResponseStream();
                    StreamReader reader = new StreamReader(dataStream, encoding);
                    responseFromServer = reader.ReadToEnd();

                    reader.Close();
                    dataStream.Close();
                    response.Close();
                    return responseFromServer;
                }
                else
                {
                    return "Error";
                }
            }
            catch (WebException e)
            {
                sException = e.Message.ToString();
                return sException;
            }
        }
        #endregion

        /// <summary>
        /// 使用 httphelper类通过url获取html
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetHtmlByHttpHelper(string url)
        {
            try
            {
                //创建Httphelper对象
                HttpHelper http = new HttpHelper();
                //创建Httphelper参数对象
                HttpItem item = new HttpItem()
                {
                    URL = url,//URL     必需项    
                    Method = "get",//URL     可选项 默认为Get   
                    ContentType = "text/html",//返回类型    可选项有默认值   
                    // ProxyIp = iplist[index],
                    Allowautoredirect = true
                    //ContentType = "application/x-www-form-urlencoded",//返回类型    可选项有默认值   
                };
                //请求的返回值对象
                HttpResult result = http.GetHtml(item);
                //获取请请求的Html
                string html = result.Html;
                return html;

            }
            catch (Exception ex)
            {

                return "";
            }
        }
        #region 异步访问url
        /// <summary>
        /// 访问地址
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static async Task<string> ScanUrl(string url)
        {
            string code = await GoToUrl(url);
            return code;
        }
        /// <summary>
        /// 访问url  代理访问
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static async Task<string> GoToUrl(string url)
        {
            //  var iplist = GetProxyByHttpHelper();
            //    string proxip = iplist[0];
            try
            {
                //创建Httphelper对象
                HttpHelper http = new HttpHelper();
                //创建Httphelper参数对象
                HttpItem item = new HttpItem()
                {
                    URL = url,//URL     必需项    
                    Method = "get",//URL     可选项 默认为Get   
                    ContentType = "text/html",//返回类型    可选项有默认值   
                    //   ProxyIp = proxip,
                    Allowautoredirect = true
                    //ContentType = "application/x-www-form-urlencoded",//返回类型    可选项有默认值   
                };
                //请求的返回值对象
                HttpResult result = http.GetHtml(item);
                //获取请请求的Html
                string code = result.StatusCode.ToString();
                return code;
            }
            catch (Exception ex)
            {
                return "";
            }
        }
        #endregion
        /// <summary>
        /// 使用代理url获取html  获取代理ip
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static List<string> GetProxyByHttpHelper()
        {
            try
            {
                string url = "http://vxer.daili666api.com/ip/?tid=555528787440920&num=3&foreign=none";
                //创建Httphelper对象
                HttpHelper http = new HttpHelper();
                //创建Httphelper参数对象
                HttpItem item = new HttpItem()
                {
                    URL = url,//URL     必需项    
                    Method = "get",//URL     可选项 默认为Get   
                    ContentType = "text/html",//返回类型    可选项有默认值   
                    // ProxyIp = proxy,
                    Allowautoredirect = true
                    //ContentType = "application/x-www-form-urlencoded",//返回类型    可选项有默认值   
                };
                //请求的返回值对象
                HttpResult result = http.GetHtml(item);
                //获取请请求的Html
                string html = result.Html;

                //获取请求的Cookie
                string cookie = result.Cookie;

                //    string getIP = "http://vxer.daili666api.com/ip/?tid=555528787440920&num=3&foreign=none";
                //  var ips = GetProxyByHttpHelper(getIP);
                var iplist = Regex.Replace(html, @"\s+", "|").Split('|').ToList();
                return iplist;
            }
            catch (Exception ex)
            {
                return null;
            }
        }
        #region Rule
        /// <summary>
        /// 正则找出需要的数据部分
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
        public static string GetContent(string html)
        {
            string first = "\"shopItems\":";
            string end = "\"apiUrl\"";
            int findex = html.IndexOf(first);
            int eindex = html.IndexOf(end);
            if (findex == -1 || eindex == -1)
            {
                return "";
            }
            var content = html.Substring(findex + first.Length, eindex - findex - first.Length - 1);
            return content;
        }

        #endregion
    }
}